#!/bin/sh

export JAVA_HOME=/app/jre1.6.0_14
#JAVA_OPTS="-Dnutch.plugin.dir=/app/nutch/plugins"
export NUTCH_OPTS="-Dnutch.plugin.dir=/app/nutch/plugins"

#The basedir for nutch installation
NUTCH_HOME=.

#Number of documents to fetch per round
NUMDOCS=1000


#Params
SERVER=$1
COLLECTION=$2
USER=$3
PASSWORD=$4
BASEDIR=$5
CYCLENUM=$6
URLSDIR=$7

if [ $# -lt 6 ]
then
echo Usage is \"crawl.sh server collection user password basedir cyclenum\"
exit
fi

HASSEG=1

checkSeg() {
 	if test "$?" = "254"; then 
           HASSEG=0
	fi
  
}

checkStatus () 
{
	if test "$?" != "0"; then 
	  echo Command exited with abnormal status, bailing out.
	  exit 1;
	fi
}

inject() 
{
echo "-------------------------------------------------"
echo "** INJECTING **"
bin/nutch inject $BASEDIR/crawldb $URLSDIR
checkStatus
echo "-------------------------------------------------"
}

nutchCycle()
{
echo "** GENERATING SEGMENT **"
bin/nutch generate $BASEDIR/crawldb $BASEDIR/segments

checkSeg
if [ "$HASSEG" = "0" ];then
	return
fi

checkStatus

echo "-------------------------------------------------"
echo "** PROCESSING SEGMENT **"
SEGMENT=$BASEDIR/segments/`ls -tr $BASEDIR/segments|tail -1`
echo processing segment $SEGMENT
echo "-------------------------------------------------"
echo "** FETCHING **"
bin/nutch fetch $SEGMENT -noParsing
checkStatus
echo "-------------------------------------------------"
echo "** PARSING **"
bin/nutch parse $SEGMENT
checkStatus
echo "-------------------------------------------------"
echo "** UPDATEDB **"
bin/nutch updatedb $BASEDIR/crawldb $SEGMENT -filter -normalize
checkStatus
count=`expr $count + 1`
}

invertLinks() 
{
echo "-------------------------------------------------"
echo "** INVERTLINKS **"
bin/nutch invertlinks $BASEDIR/linkdb -dir $BASEDIR/segments
checkStatus
echo "-------------------------------------------------"
}

indexToSolr() 
{
echo "** INDEXING TO SOLR **"

bin/nutch br.com.abril.hubble.nutch.indexer.HubbleIndexer $SERVER $COLLECTION $USER $PASSWORD $BASEDIR/crawldb $BASEDIR/linkdb $BASEDIR/segments/*

checkStatus
echo "-------------------------------------------------"
}

cd $NUTCH_HOME

echo "*************************************************"

inject

# generate/fetch/update cycle
count=0								
while [ $count -lt $CYCLENUM ]	
do
	nutchCycle
	if [ "$HASSEG" = "0" ];then
		count=`expr $count + 1`
		continue
	fi
done

invertLinks

indexToSolr

echo "************** DONE *****************************"



